package com.laivi.basic.module.crawler;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.laivi.basic.common.constants.Constant;
import com.laivi.basic.model.crawler.BloomFilter;


public class SaveUrlDoc implements Runnable {
	private LinkedBlockingQueue<String> parsereddQueue;
	private BloomFilter filter;
	private CountDownLatch latch;
	
	/**
	 * @param parsereddQueue
	 * @param filter
	 */
	public SaveUrlDoc(LinkedBlockingQueue<String> parsereddQueue,
			BloomFilter filter,CountDownLatch latch) {
		super();
		this.parsereddQueue = parsereddQueue;
		this.filter = filter;
		this.latch=latch;
	}

	@Override
	public void run() {
		while(filter.getSize()<Constant.Crawler.MAX){
			System.out.println("filter Size:"+filter.getSize());
			try{
				String url=parsereddQueue.take();
				System.out.println("pasered remove Url:"+url);
				if(!filter.contains(url)){
					System.out.println(url);
					Document doc =Jsoup.connect(url).ignoreContentType(true).cookie("auth","token")
							.header("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2")
							.timeout(Constant.Crawler.TIME_OUT).get();
					String filePath=parseUrlToDir(Constant.Crawler.FILE_PATH,url);
					FileOutputStream out =new FileOutputStream(new File(filePath),false);
					out.write(doc.html().getBytes(Constant.Encode.UTF8));
					out.flush();
					out.close();
					filter.add(url);
				}
			}catch(Exception e){
				e.printStackTrace();
			}
		}
		latch.countDown();
	}
	
	public String parseUrlToDir(String path,String url){
		String temp=url.replaceAll("http://", "").replaceAll("https://", "");
		if(temp.indexOf("?")!=-1){
			temp=temp.substring(0, temp.indexOf("?"));
		}
		if(temp!=null&& !"".equals(temp)){
			return createDir(path,temp.split("/"));
		}else{
			return null;
		}
	}
	
	public String createDir(String path,String[] names){
		StringBuffer sb=new StringBuffer(path);
		int i=0;
		String filePath=UUID.randomUUID().toString()+".txt";
		for(String name:names){
			if(i==0){
				if(name.indexOf(":")!=-1){
					name=name.substring(0, name.indexOf(":"));
				}
			}
			if(i!=(names.length-1)||names.length==1){
				sb.append("\\"+name);
			}
			i++;
		}
		File dir=new File(sb.toString());
		if(!dir.exists()){
			dir.mkdirs();
		}
		File file=new File(sb.toString()+"\\"+filePath);
		if(!file.exists()){
			try {
				file.createNewFile();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return sb.toString()+"\\"+filePath;
	}

}
